import os
import random

from gxl_ai_utils.utils import utils_file

input_all_text_path = '/home/work_nfs8/xlgeng/new_workspace/gxl_ai_utils/eggs/cats_and_dogs/ngram_task/data_handler/haha/all_data-nospace-ABC.txt'
output_dir_root = './gxl_data_zoo/all_data_new'
utils_file.makedir_sil(output_dir_root)
# all_text_list = utils_file.load_list_file_clean(input_all_text_path)
# random.shuffle(all_text_list)
# list_list = utils_file.do_split_list(all_text_list,100)
# for i,list_i in enumerate(list_list):
#     output_dir_temp = utils_file.join_path(output_dir_root, "temp", f"partition_{i}")
#     utils_file.write_list_to_file(list_i, utils_file.join_path(output_dir_temp,"input.list"))

# runner = utils_file.GxlDynamicThreadPool()
# for i in range(0, 10):
#     output_dir_temp = utils_file.join_path(output_dir_root, "temp", f"partition_{i}")
#     input_list_path = os.path.join(output_dir_temp, 'input.list')
#     command_line = f"python main.py {input_list_path} {output_dir_temp}"
#     runner.add_task(utils_file.do_execute_shell_command, [command_line])
# runner.start()
res_list = []
for i in range(0, 100):
    output_dir_temp = utils_file.join_path(output_dir_root, "temp", f"partition_{i}")
    input_list_path = os.path.join(output_dir_temp, 'final_data.list')
    if os.path.exists(input_list_path):
        temp_list = utils_file.load_list_file_clean(input_list_path)
        res_list.extend(temp_list)

random.shuffle(res_list)
utils_file.write_list_to_file(res_list, utils_file.join_path(output_dir_root, 'final_data.list'))
